package jerry.file.demo;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import java.util.TreeMap;
import java.util.TreeSet;

/**
 * @author test
 *
 */
public class KeyWordsHelper {

	
	HashSet<String> otherSet = new HashSet<String>(120503);
	HashSet<String> baseSetAndOther = new HashSet<String>(2620000);
	
	
	List<List<String>> list = new ArrayList<List<String>>(12);
	
	int[] lengths = {34, 200852, 468472, 1006880, 555354, 10000, 10000, 10000, 10000, 10000, 10000, 10000};
	
	private void initList(){
		for(int i=0;i<lengths.length; i++){
			 ArrayList<String> arrayList = new ArrayList<String>(lengths[i]);
			 list.add(arrayList);
		}
	}
	
	private void initSets() throws IOException{
		otherSet.clear();
		addOtherKeyWordsToSet("place.txt", otherSet);
		addOtherKeyWordsToSet("org.txt", otherSet);
		addOtherKeyWordsToSet("people.txt", otherSet);
		addOtherKeyWordsToSet("人名.txt", otherSet);
		addOtherKeyWordsToSet("机构.txt", otherSet);
		addOtherKeyWordsToSet("地名.txt", otherSet);
	}
	
	private void initAllSet()  throws IOException{
		baseSetAndOther.clear();
//		addOtherKeyWordsToSet("place.txt", baseSetAndOther);
//		addOtherKeyWordsToSet("org.txt", baseSetAndOther);
		addOtherKeyWordsToSet("keywords.txt", baseSetAndOther);
	}
	
	private void addOtherKeyWordsToSet(String filePath, Set<String> set) throws IOException{
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\"+filePath, "utf-8");
		String currentString;
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			if(set.contains(currentString))	continue;
			set.add(currentString);
		}
		System.out.println("otherSet size:"+set.size());
	}
	

	/**
	 * 去重复
	 * @throws IOException
	 */
	public void noDump() throws IOException{
		
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\temp\\keywords.txt", "utf-8");
		
		HashSet<String> set = new HashSet<String>(38200904);
		
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamOrg = new FileOutputStream("E:\\data\\不重复\\temp\\keywords_noDum.txt");
		BufferedWriter bufferedWriterOrg = new BufferedWriter(new OutputStreamWriter(fileOutputStreamOrg, "utf-8"));
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(count % 10000 == 0){
				System.out.println("读取了："+count+"条。");
			}
			currentString = trimStr(currentString);
			if(set.contains(currentString))	continue;
			set.add(currentString);
			bufferedWriterOrg.write(currentString);
			bufferedWriterOrg.write("\n");
		}
		bufferedWriterOrg.close();
		bufferedReader.close();
	
	}
	
	/**
	 * 去除地区、人物、组织
	 * @throws IOException
	 */
	public void removeOrgPeoPlace() throws IOException{
		
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords.txt", "utf-8");
		
		initSets();
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamOrg = new FileOutputStream("E:\\data\\不重复\\keywords_clean.txt");
		BufferedWriter bufferedWriterOrg = new BufferedWriter(new OutputStreamWriter(fileOutputStreamOrg, "utf-8"));
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(count % 10000 == 0){
				System.out.println("读取了："+count+"条。");
			}
			if(otherSet.contains(currentString)){
				System.out.println(currentString);
				continue;
			}
			bufferedWriterOrg.write(currentString);
			bufferedWriterOrg.write("\n");
		}
		bufferedWriterOrg.close();
		bufferedReader.close();
	}
	
	/**
	 * 清楚不合法的数据，如果  魏·《的  、9·16等
	 * @throws IOException 
	 */
	private void clean() throws IOException{
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords.txt", "utf-8");
		FileOutputStream fileOutputStreamOrg = new FileOutputStream("E:\\data\\不重复\\keywords_baddata.txt");
		BufferedWriter bufferedWriterOrg = new BufferedWriter(new OutputStreamWriter(fileOutputStreamOrg, "utf-8"));
		FileOutputStream fileOutputStreamClean = new FileOutputStream("E:\\data\\不重复\\keywords_clean.txt");
		BufferedWriter bufferedWriterClean = new BufferedWriter(new OutputStreamWriter(fileOutputStreamClean, "utf-8"));
		String currentString;
		long count = 0;
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(count % 10000 == 0){
				System.out.println("读取了："+count+"条。");
			}
			if(isBadData(currentString)){
				bufferedWriterOrg.write(currentString);
				bufferedWriterOrg.write("\n");				
			}else{
				bufferedWriterClean.write(currentString);
				bufferedWriterClean.write("\n");	
			}
		}
		bufferedWriterOrg.close();
		bufferedReader.close();
		bufferedWriterClean.close();
	}
	
	private boolean isBadData(String temp){
		char firstChar = temp.charAt(0);
		char endChar = temp.charAt(temp.length()-1);
		if(endChar =='《')	return true;
		if(firstChar =='》')	return true;
		if(endChar == '“')	return true;
		return isAllNumOrEngOrDot(temp);
	}
	
	/**
	 * 所有字符全是数字和字符
	 * @param temp
	 * @return
	 */
	private boolean isAllNumOrEngOrDot(String temp){
		boolean haveNumOrEng = false;
		int size = temp.length();
		for(int i=0;i<size;i++){
			char c = temp.charAt(i);
//			if(isChinese(c))	return false;
			if(isSomeChar(c))	return false;
			if(isNumOrEng(c)||isDot(c)){
				haveNumOrEng = true;
			}else{
				return false;
			}
		}
		return haveNumOrEng;
	}
	
	private boolean isNumOrEng(String temp){
		boolean haveNumOrEng = false;
		int size = temp.length();
		for(int i=0;i<size;i++){
			char c = temp.charAt(i);
			if(isChinese(c))	return false;
			if(isSomeChar(c))	return false;
			if(isNumOrEng(c)){
				haveNumOrEng = true;
			}
		}
		return haveNumOrEng;
	}
	
	
	/**
	 * 如果是特殊字符则不是脏数据
	 * @param c
	 * @return
	 */
	private boolean isSomeChar(char c){
		return c=='“';
	}
	
	private boolean isChinese(char c){
		return c>=19968;
	}
	
	private boolean isNumOrEng(char c){
		return (c < 126 && c>0)||isCharNum(c);
	}
	
	private boolean isDot(char c){
		return c=='·';
	}
	
	/**
	 * 按照字数多少进行排序
	 * @throws IOException
	 */
	public void sortByLength() throws IOException{
		
		initList();
		
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords_clean.txt", "utf-8");
		
		
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamOrg = new FileOutputStream("E:\\data\\不重复\\keywords_sort.txt");
		BufferedWriter bufferedWriterOrg = new BufferedWriter(new OutputStreamWriter(fileOutputStreamOrg, "utf-8"));
		
		FileOutputStream fileOutputStreamMore = new FileOutputStream("E:\\data\\不重复\\keywords_sortMore.txt");
		BufferedWriter bufferedWriterMore = new BufferedWriter(new OutputStreamWriter(fileOutputStreamMore, "utf-8"));
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(count % 10000 == 0){
				System.out.println("读取了："+count+"条。");
			}
			if(currentString.length() > 12){
				bufferedWriterMore.write(currentString);
				bufferedWriterMore.write("\n");
			}else{
				List<String> tempList = list.get(currentString.length()-1);
				tempList.add(currentString);
			}
//			bufferedWriterOrg.write(currentString);
//			bufferedWriterOrg.write("\n");
			
			
		}
		
		for(List<String> tempList : list){
			for(String keyWord : tempList){
				bufferedWriterOrg.write(keyWord);
				bufferedWriterOrg.write("\n");
			}
		}
		
		bufferedWriterOrg.close();
		bufferedReader.close();
		fileOutputStreamMore.close();
		
	}
	
	public void sortByLength2() throws IOException{
		
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords.txt", "utf-8");
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamOrg = new FileOutputStream("E:\\data\\不重复\\keywords_sort.txt");
		BufferedWriter bufferedWriterOrg = new BufferedWriter(new OutputStreamWriter(fileOutputStreamOrg, "utf-8"));
		
		TreeSet<String> set = new TreeSet<String>(new Comparator<String>() {

			@Override
			public int compare(String o1, String o2) {
				int temp = o1.length() - o2.length();
				if(temp > 0){
					return 1;
				}else{
					return -1;
				}
			}
		});
		
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(count % 10000 == 0){
				System.out.println("读取了："+count+"条。");
			}
			set.add(currentString);
			
		}
		
		System.out.println("set size:"+set.size());
		System.out.println("read line:"+count);
		Iterator<String> iterator = set.iterator();
		while(iterator.hasNext()){
			bufferedWriterOrg.write(iterator.next());
			bufferedWriterOrg.write("\n");
		}
		
		bufferedWriterOrg.close();
		bufferedReader.close();
	}
	
	private String trimStr(String temp){
		int length = temp.length();
		if(temp.charAt(0) == '《' && temp.charAt(length-1) == '》'){
			return temp.substring(1, length-1);
		}else if(temp.charAt(0) == '“' && temp.charAt(length-1) == '”'){
			return temp.substring(1, length-1);
		}
		return temp;
	}
	
	private void removePeople() throws IOException{
		
		initAllSet();
		
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords.txt", "utf-8");
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamPeople = new FileOutputStream("E:\\data\\不重复\\forgPeople.txt");
		BufferedWriter bufferedWriterPeople = new BufferedWriter(new OutputStreamWriter(fileOutputStreamPeople, "utf-8"));
		FileOutputStream fileOutputStreamRemovePeople = new FileOutputStream("E:\\data\\不重复\\keywordsRemovePeople.txt");
		BufferedWriter bufferedWriterRemovePeople = new BufferedWriter(new OutputStreamWriter(fileOutputStreamRemovePeople, "utf-8"));
		
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
//			if(count % 10000 == 0){
//				System.out.println("读取了："+count+"条。");
//			}
			if(isPeople(currentString)){
				bufferedWriterPeople.write(currentString);
				bufferedWriterPeople.write("\n");	
//				if(currentString.length() > 3){
//					StringTokenizer token = new StringTokenizer(currentString, "·");
//					StringBuilder sb = new StringBuilder();
//					while(token.hasMoreTokens()){
//						String nextToken = token.nextToken();
//						if(baseSetAndOther.contains(nextToken)){
//							System.out.println(currentString);
//							break;
//						}
//						if(nextToken.length()>1){
//							if(isNotName(nextToken)){
//								sb.append(nextToken).append("\n");
////								System.out.println(nextToken);
////								bufferedWriterPeople.write(nextToken);
////								bufferedWriterPeople.write("\n");															
//							}else{
//							}
//						}
//					}
//					bufferedWriterPeople.write(sb.toString());
//				}else{
////					System.out.println(currentString);
//				}
			}else{
				bufferedWriterRemovePeople.write(currentString);
				bufferedWriterRemovePeople.write("\n");
			}
		}
		
		bufferedWriterPeople.close();
		bufferedReader.close();
		bufferedWriterRemovePeople.close();
	}
	
	/**
	 * 移出带点的词
	 * @throws IOException
	 */
	private void removeDot() throws IOException{
		
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords.txt", "utf-8");
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamPeople = new FileOutputStream("E:\\data\\不重复\\withdot.txt");
		BufferedWriter bufferedWriterPeople = new BufferedWriter(new OutputStreamWriter(fileOutputStreamPeople, "utf-8"));
		FileOutputStream fileOutputStreamRemovePeople = new FileOutputStream("E:\\data\\不重复\\keywordsRemoveDot.txt");
		BufferedWriter bufferedWriterRemovePeople = new BufferedWriter(new OutputStreamWriter(fileOutputStreamRemovePeople, "utf-8"));
		
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(withDot(currentString)){
				bufferedWriterPeople.write(currentString);
				bufferedWriterPeople.write("\n");	
			}else{
				bufferedWriterRemovePeople.write(currentString);
				bufferedWriterRemovePeople.write("\n");
			}
		}
		
		bufferedWriterPeople.close();
		bufferedReader.close();
		bufferedWriterRemovePeople.close();
	}
	
	/**
	 * 移出组织
	 * @throws IOException
	 */
	private void removeOrg() throws IOException{
		
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords.txt", "utf-8");
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamPeople = new FileOutputStream("E:\\data\\不重复\\org\\org.txt");
		BufferedWriter bufferedWriterPeople = new BufferedWriter(new OutputStreamWriter(fileOutputStreamPeople, "utf-8"));
		FileOutputStream fileOutputStreamRemovePeople = new FileOutputStream("E:\\data\\不重复\\keywordsRemoveOrg.txt");
		BufferedWriter bufferedWriterRemovePeople = new BufferedWriter(new OutputStreamWriter(fileOutputStreamRemovePeople, "utf-8"));
		
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(isOrg(currentString)){
				bufferedWriterPeople.write(currentString);
				bufferedWriterPeople.write("\n");	
			}else{
				bufferedWriterRemovePeople.write(currentString);
				bufferedWriterRemovePeople.write("\n");
			}
		}
		
		bufferedWriterPeople.close();
		bufferedReader.close();
	}
	String[] orgcontains = {"会","局","社","所","公司","部","协会","队","研究院","办公室","中心","小学","大学","工作小组","支行","联盟"
			,"党委","办公厅","厂","办事处","研究室","文工团","政府","学校","主席团","筹备处","政府","同盟","小组","实验室","歌舞团","分院"
			,"技术科","矿场","农场","中学","基地","卫生院","学院","合唱团","工作站","集团","大使馆","书记处","管理处","监察处",
			"管理科"};
	
	String[] notOrgs = {"干部","晚会","格局","团队","舰队","团队","社会","场所","座谈会","演唱会"};
	
	private boolean isOrg(String temp){
		if(temp.length()<7)	return false;
		for(String org : notOrgs){
			if(temp.endsWith(org))	return false;
		}
		for(String org : orgcontains){
			if(temp.endsWith(org))	return true;
		}
		return false;
	}
	
	String[] places = {"馆"};
	private boolean isPlace(String temp){
		for(String place : places){
			if(temp.endsWith(place))	return true;
		}
		return false;
	}
	
	private boolean isNotName(String name){
		for(int i=0;i<name.length();i++){
			char c = name.charAt(i);
			if(c>=65296 && c<=65305)	return true;
		}
		return false;
	}
	
	/**
	 * 是否有特殊字符
	 * @param temp
	 * @return
	 */
	private boolean haveSpeChar4RemovePeople(String temp){
		for(int i=0;i<temp.length();i++){
			char c=temp.charAt(i);
			if(c=='“' || c<64 || isChineseNum(c) || isCharNum(c)){
//				System.out.println(temp);
				return true;
			}
//			return c=='“';
		}
		return false;
	}
	
	private boolean isChineseNum(char c){
		switch(c){
		case 19968: case 20108:case 19977:case 22235:case 20116:case 20845:case 19971:case 20843:case 20061:case 21313:
			return true;
		}
		return false;
	}
	
	private boolean isCharNum(char c){
		return c>=65296&&c<=65305;
	}
	
	private void removeEnglish() throws IOException{
		BufferedReader bufferedReader = FileUtil.getBufferedReader("E:\\data\\不重复\\keywords.txt", "utf-8");
		String currentString;
		long count = 0;
		FileOutputStream fileOutputStreamPeople = new FileOutputStream("E:\\data\\不重复\\eng.txt");
		BufferedWriter bufferedWriterPeople = new BufferedWriter(new OutputStreamWriter(fileOutputStreamPeople, "utf-8"));
		
		while (true) {
			currentString = bufferedReader.readLine();
			if (currentString == null) break;
			count++;
			if(count % 10000 == 0){
				System.out.println("读取了："+count+"条。");
			}
			if(isPeople(currentString)){
				bufferedWriterPeople.write(currentString);
				bufferedWriterPeople.write("\n");
			}
		}
		
		bufferedWriterPeople.close();
		bufferedReader.close();
		
	
	}
	
	private boolean isPeople(String temp){
		return temp.indexOf('·') >=0 && !haveSpeChar4RemovePeople(temp);
	}
	
	private boolean withDot(String temp){
		return temp.indexOf('·') >=0;
	}
	
	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		KeyWordsHelper keyWordsHelper = new KeyWordsHelper();
//		keyWordsHelper.removeOrgPeoPlace();
//		keyWordsHelper.removeDot();
		keyWordsHelper.removeOrg();
//		keyWordsHelper.clean();
//		System.out.println((int)'９');
////		System.out.println((int)'Ａ');
//		for(int i=65315;i<65415;i++){
//			char c = (char) i;
//			System.out.println(i+":"+c);
//		}
	}

}
